<!DOCTYPE html>



  


<html class="theme-next muse use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222">





  <meta name="keywords" content="Hexo, NexT" />










<meta name="description" content="Canal 的 dump 支持串行和并行模式两种模式，本篇重点梳理 dump 的核心流程，以便对 dump 过程有一个充分的了解，更好的理解 Canal 的实现原理与细节，下一篇中将重点关注Canal是如何引入并行模式来提高dump的性能，即并行编程相关的技巧。 从前面的文章我们得知 Canal binlog 日志解析的基本流程如下图所示：解析来重点梳理一下 dump 命令的发送逻辑，特别是日志的">
<meta property="og:type" content="article">
<meta property="og:title" content="Canal binlog 日志 Dump 流程分析">
<meta property="og:url" content="https://www.codingw.net/posts/aeb9a46c.html">
<meta property="og:site_name" content="中间件兴趣圈">
<meta property="og:description" content="Canal 的 dump 支持串行和并行模式两种模式，本篇重点梳理 dump 的核心流程，以便对 dump 过程有一个充分的了解，更好的理解 Canal 的实现原理与细节，下一篇中将重点关注Canal是如何引入并行模式来提高dump的性能，即并行编程相关的技巧。 从前面的文章我们得知 Canal binlog 日志解析的基本流程如下图所示：解析来重点梳理一下 dump 命令的发送逻辑，特别是日志的">
<meta property="og:locale">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175601739.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175612589.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175625678.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175645466.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175657119.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175707476.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175717357.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175726519.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175748490.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175803860.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175816470.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175830366.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175842606.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/202008091758523.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/2020080917590177.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175910291.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/2020080917592088.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175932873.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175945791.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200809175956839.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="article:published_time" content="2020-11-20T14:05:01.000Z">
<meta property="article:modified_time" content="2021-04-26T12:13:04.646Z">
<meta property="article:author" content="中间件兴趣圈">
<meta property="article:tag" content="中间件">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/20200809175601739.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '',
    scheme: 'Muse',
    version: '5.1.4',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="https://www.codingw.net/posts/aeb9a46c.html"/>





  <title>Canal binlog 日志 Dump 流程分析 | 中间件兴趣圈</title>
  








<meta name="generator" content="Hexo 5.4.0"></head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">中间件兴趣圈</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">微信搜『中间件兴趣圈』，回复『Java』获取200本优质电子书</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-question-circle"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-question-circle"></i> <br />
            
            归档
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://www.codingw.net/posts/aeb9a46c.html">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="中间件兴趣圈">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Canal binlog 日志 Dump 流程分析</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2020-11-20T22:05:01+08:00">
                2020-11-20
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/canal/" itemprop="url" rel="index">
                    <span itemprop="name">canal</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          
             <span id="/posts/aeb9a46c.html" class="leancloud_visitors" data-flag-title="Canal binlog 日志 Dump 流程分析">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv"><i class="fa fa-file-o"></i>
            <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>次
            </span>
          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <div id="vip-container"><p>Canal 的 dump 支持串行和并行模式两种模式，本篇重点梳理 dump 的核心流程，以便对 dump 过程有一个充分的了解，更好的理解 Canal 的实现原理与细节，下一篇中将重点关注Canal是如何引入并行模式来提高dump的性能，即并行编程相关的技巧。</p>
<p>从前面的文章我们得知 Canal binlog 日志解析的基本流程如下图所示：<br><img src="https://img-blog.csdnimg.cn/20200809175601739.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>解析来重点梳理一下 dump 命令的发送逻辑，特别是日志的处理流程，一些基本的日志格式。</p>
<h2 id="1、-dump-流程分析"><a href="#1、-dump-流程分析" class="headerlink" title="1、 dump 流程分析"></a>1、 dump 流程分析</h2><p>在 Canal 中 dump 方法声明如下：<br><img src="https://img-blog.csdnimg.cn/20200809175612589.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>带有参数 MultiStageCoprocessor 为并行处理模式，底层使用了disruptor 高性能并发框架，下文将重点关注学习。我们今天重点来看一下串行dump的实现，其方法声明如下：<img src="https://img-blog.csdnimg.cn/20200809175625678.png#pic_center" alt="在这里插入图片描述">其方法参数说明如下：</p>
<ul>
<li>String binlogfilename<pre><code> binlog 文件名称，例如  mysql-bin.000038。
</code></pre>
</li>
<li>Long binlogPosition<br>在文件中的偏移量。</li>
<li>SinkFunction func<br>每解析出一条binlog日志的处理函数。</li>
</ul>
<p>接下来我们直奔主题，一起来看一下 MysqlConnection 关于 dump 的实现流程。</p>
<span id="more"></span>

<p><img src="https://img-blog.csdnimg.cn/20200809175645466.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="MysqlConnection#dump"><br>Step1：在发送dump之前先设置相关的参数。</p>
<ul>
<li>set wait_timeout=9999999<br>连接空闲超时时间，默认为8消息，用于 Canal Slave 的等待超时时间远大于默认值。</li>
<li>set net_write_timeout=1800<br>网络写请求超时时间，针对正在进行数据读写的连接，该值默认为 60s。</li>
<li>set net_read_timeout=1800<br>网络读请求超时时间，针对正在进行数据读写的连接，该值默认为 30s。</li>
<li>set names ‘binary’<br>设置服务端返回结果时不做编码转化，直接按照数据库的二进制编码进行发送，由客户端自己根据需求进行编码转化。</li>
<li>set @master_binlog_checksum= @@global.binlog_checksum<br>设置master_binlog_checksum，因为在mysql5.6之后为binlog引入了checksum机制，例如crc32，canal作为mysql slave，需要与服务端相关参数保持一致。</li>
<li>set @slave_uuid=uuid()<br>canal相对与mysql数据库服务而言就是一个从服务器，这个指令用于设置server_id，使用uuid，避免server_id重复。</li>
<li>SET @master_heartbeat_period=15<br>设置客户端与服务端心跳发送间隔，默认为15s。<br><img src="https://img-blog.csdnimg.cn/20200809175657119.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="MysqlConnection#dump"><br>Step2：从主库查询binlog checksum，具体向主库发送 select @@global.binlog_checksum 语句。<br><img src="https://img-blog.csdnimg.cn/20200809175707476.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="MysqlConnection#dump"><br>Step3：向MySQL Master 注册从节点，告知客户端的host、port、用户名与密码、serverId，具体实现是发送命令CODE为 0x15。<br><img src="https://img-blog.csdnimg.cn/20200809175717357.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="MysqlConnection#dump"><br>Step4：向 MySQL Master 发送 dump 请求，MySQL是基于请求与应答模式，发送请求命令后，就会向网络通道中写入响应请求。（在这里大家不妨先大概思考一下如何读取 dump 命令的返回值，这部分虽然涉及到网络相关的知识，我在这边会稍微简单提一下）。<br><img src="https://img-blog.csdnimg.cn/20200809175726519.png#pic_center" alt="MysqlConnection#dump"><br>Step5：构建 DirectLogFetcher对象，实现基于 socket 的日志拉取服务，并构建 LogDecoder 对象，用于解析 binlog 日志。<br><img src="https://img-blog.csdnimg.cn/20200809175748490.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="MysqlConnection#dump"><br>Step6：使用 while 循环反复拉取消息，通过通过 LogDecoder 对二进制流进行解析，提取一条完整的binlog事件，交给 SinkFunction 去处理，并且如果开启了半同步机制，则需要向master发送ACK。既然是while循环，该方法的退出条件还是值的我们关注的：</li>
<li>fetch.fetch()方法返回 false</li>
<li>SinkFunction 的 sink 方法 false，SinkFunction的详细处理流程将在下文介绍，这里先告知返回false的情况是 binlog 日志解析线程已停止运行。</li>
</ul>
<p>上面粗略的介绍了 dump 命令的几个核心关键步骤，要想详细掌握其实现细节，我们必须继续深入探讨如下几个问题：</p>
<ul>
<li>DirectLogFetcher 内部工作机制</li>
<li>LogDecoder binlog 日志解析</li>
<li>发送Dump底层网实现思路</li>
</ul>
<h2 id="2、DirectLogFetcher-内部工作机制"><a href="#2、DirectLogFetcher-内部工作机制" class="headerlink" title="2、DirectLogFetcher 内部工作机制"></a>2、DirectLogFetcher 内部工作机制</h2><h4 id="2-1-DirectLogFetcher-类图"><a href="#2-1-DirectLogFetcher-类图" class="headerlink" title="2.1 DirectLogFetcher 类图"></a>2.1 DirectLogFetcher 类图</h4><p><img src="https://img-blog.csdnimg.cn/20200809175803860.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>DirectLogFetcher的类继承体系如上图所示，我们来看一下其关键点：</p>
<ul>
<li>LogBuffer<br>日志buffer，主要定义如下属性：</li>
<li>byte[] buffer<br>缓存区中数据容器。</li>
<li>int origin<br>当前buffer中的读指针</li>
<li>int limit<br>当前buffer的最大可读可写指针</li>
<li>int position<br>当前buffer的写指针。</li>
<li>int semival<br>是否需要发送ACK(用于半同步)。<br>LogBuffer封装了字节相关的操作，不仅定义了上面的属性，也定义了字节读取相关众多API，其截图如下：<br><img src="https://img-blog.csdnimg.cn/20200809175816470.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"></li>
<li>LogFetcher<br>binlog日志抓取抽象类，定义了如下关键属性与抽象方法。<ul>
<li>int DEFAULT_INITIAL_CAPACITY<br>LogBuffer中的初始容量，默认为8K。</li>
<li>float DEFAULT_GROWTH_FACTOR<br>容量增长因子，默认为 2.0。</li>
<li>int   BIN_LOG_HEADER_SIZE<br>binlog日志条目 header 的长度，固定为4字节。</li>
<li>float  factor<br>  增长因子。</li>
<li>public abstract boolean fetch()<br>抓取binlog日志。</li>
<li>public abstract void close() <pre><code>    关闭 Fetch。
</code></pre>
</li>
</ul>
</li>
<li>DirectLogFetcher<br>Canal LogFetcher模式实现类，其核心属性如下：<ul>
<li>SocketChannel channel<br>网络通道，用于发送dump请求的网络通道。</li>
<li>boolean issemi = false<br>是否开启半同步。</li>
</ul>
</li>
</ul>
<h4 id="2-2-fetch流程详解"><a href="#2-2-fetch流程详解" class="headerlink" title="2.2 fetch流程详解"></a>2.2 fetch流程详解</h4><p>接下来我们重点剖析 DirectLogFetcher 的 fetch 方法，来探究其实现原理。<br>在研究DirectLogFetcher的fetch方法之前，我们先重点跟踪一下其内部网络读写方法fetch0方法，该方法是具体与网络读写相关的实现。<br><img src="https://img-blog.csdnimg.cn/20200809175830366.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="DirectLogFetcher#fetch0"><br>在详细介绍该方法之前先来介绍一下其参数的含义：</p>
<ul>
<li>int off<br>从通道中读取到的内容放入到buffer中的起始位置</li>
<li>int len<br>期望从通道中读取的字节长度。</li>
</ul>
<p>该方法的实现关键点如下：</p>
<ul>
<li>首先先确保接收缓存区有足够的剩余空间，如果空间不足，则进行扩容。</li>
<li>然后从通道中读取指定长度的字节。</li>
</ul>
<p>接下来我们来重点看一下DirectLogFetcher的fetch的实现流程。<br><img src="https://img-blog.csdnimg.cn/20200809175842606.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="DirectLogFetcher#fetch"><br>Step1：尝试从网络通道中读取４个字节（即读取协议的头部），如果通道中还没有可读取内容，返回false，造成的效果是一次 dump 请求结束。<br><img src="https://img-blog.csdnimg.cn/202008091758523.png#pic_center" alt="DirectLogFetcher#fetch"><br>Step2：从上文读到的４个字节分别读出该网络包的总长度以及当前包的序号，从这里可以看成MySQL协议头为４字节，前３个字节为网络包的总长度，第４个字节为包的序列号。再取出数据包的长度后，继续向通道中读取netlen个字节，即读取一个完整的数据包到buffer中。<br><img src="https://img-blog.csdnimg.cn/2020080917590177.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="DirectLogFetcher#fetch"><br>Step3：继续从数据包中读取一个字节，判断该包的状态码，是否是一个成功的响应，如果是错误的响应，会向外抛出一次，Canal 会记录dump命令执行错误的次数。<br><img src="https://img-blog.csdnimg.cn/20200809175910291.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="DirectLogFetcher#fetch"><br>Step４：如果一个包的长度为允许的最大包长度，则继续读取，这个主要是根据MySQL协议做的处理，即读取到一个数据包，然后返回true，表示拉取到一条日志，然后通过LogDecoder解码，然后传入到sink方法中，进行日志的后续处理。<br><img src="https://img-blog.csdnimg.cn/2020080917592088.png#pic_center" alt="DirectLogFetcher#fetch"><br>Step5：这一步的目的，就是将buffer中的当前指针指向数据的开始位置。这样一次 fetch就结束了。</p>
<p>从上面的流程来看，DirectLogFetcher#fetch 方法结束后，就将进入到LogDecoder中。经过一次DirectLogFetcher#fetch方法后，即取回一条binlog日志，即二进制流，接下来就根据binlog协议对其解析。本文暂不深入该方法，如果大家想深入数据库中间件方面，可以作为一个很好的示例，面向MySQL通信协议进行编程。</p>
<h2 id="３、SinkFunction"><a href="#３、SinkFunction" class="headerlink" title="３、SinkFunction"></a>３、SinkFunction</h2><p>通过 LogDecoder从中解析一个事件后，会调用SinkFunction的sink方法，如果该方法返回 false，一次dump请求将介绍，接下来我们看一下其sink方法。<br><img src="https://img-blog.csdnimg.cn/20200809175932873.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="AbstractEventParser＃start"><br>该方法的实现比较简单，这里不打算继续深入，我们重点来看一下 Canal.Entry 的结构：<br><img src="https://img-blog.csdnimg.cn/20200809175945791.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>这个结构是基于Canal做架构设计，解决顺序消费、数据不丢失一个重要参考依据，没解析一条事务，最终放如到环形缓存区，环形缓存区尽量以一个事务提交到Sink组件，其代码如下：<br><img src="https://img-blog.csdnimg.cn/20200809175956839.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>这里主要有如下几个关键点：</p>
<ul>
<li>首先需要调用EventSink组件将解析出来的数据传入EventSink。</li>
<li>EventSink组件处理成功后，会提交解析位点。</li>
</ul>
<p>本文就介绍到这里了，Dump的流程就分析到这里了。</p>
</div>

			<script src="https://my.openwrite.cn/js/readmore.js" type="text/javascript"></script>
			<script>
			var isMobile = navigator.userAgent.match(/(phone|pad|pod|iPhone|iPod|ios|iPad|Android|Mobile|BlackBerry|IEMobile|MQQBrowser|JUC|Fennec|wOSBrowser|BrowserNG|WebOS|Symbian|Windows Phone)/i);
			if (!isMobile) {
			    var btw = new BTWPlugin();
			    btw.init({
			        "id": "vip-container",
			        "blogId": "18019-1573088808868-542",
			        "name": "中间件兴趣圈",
			        "qrcode": "https://img-blog.csdnimg.cn/20190314214003962.jpg",
			        "keyword": "more"
			    });
			}
			</script>
		
      
    </div>
    
    
    

    

    

    

    <footer class="post-footer">
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/posts/70022ae6.html" rel="next" title="源码分析Dubbo异步调用与事件回调机制">
                <i class="fa fa-chevron-left"></i> 源码分析Dubbo异步调用与事件回调机制
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/posts/c1c80d83.html" rel="prev" title="源码分析Dubbo序列化-源码分析kryo序列化实现原理">
                源码分析Dubbo序列化-源码分析kryo序列化实现原理 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <p class="site-author-name" itemprop="name"></p>
              <p class="site-description motion-element" itemprop="description"></p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/archives/">
              
                  <span class="site-state-item-count">139</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">18</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            

          </nav>

          

          

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#1%E3%80%81-dump-%E6%B5%81%E7%A8%8B%E5%88%86%E6%9E%90"><span class="nav-number">1.</span> <span class="nav-text">1、 dump 流程分析</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2%E3%80%81DirectLogFetcher-%E5%86%85%E9%83%A8%E5%B7%A5%E4%BD%9C%E6%9C%BA%E5%88%B6"><span class="nav-number">2.</span> <span class="nav-text">2、DirectLogFetcher 内部工作机制</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#2-1-DirectLogFetcher-%E7%B1%BB%E5%9B%BE"><span class="nav-number">2.0.1.</span> <span class="nav-text">2.1 DirectLogFetcher 类图</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#2-2-fetch%E6%B5%81%E7%A8%8B%E8%AF%A6%E8%A7%A3"><span class="nav-number">2.0.2.</span> <span class="nav-text">2.2 fetch流程详解</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%EF%BC%93%E3%80%81SinkFunction"><span class="nav-number">3.</span> <span class="nav-text">３、SinkFunction</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">中间件兴趣圈</span>

  
</div>


  <div class="powered-by">由 <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a> 强力驱动</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 &mdash; <a class="theme-link" target="_blank" href="https://github.com/iissnan/hexo-theme-next">NexT.Muse</a> v5.1.4</div>




        
<div class="busuanzi-count">
  <script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv">
      <i class="fa fa-user"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
      
    </span>
  

  
    <span class="site-pv">
      <i class="fa fa-eye"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
      
    </span>
  
</div>








        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

  
  
    <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.4"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.4"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script>



  


  




	





  





  












  





  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.4.js"></script>
  <script>AV.initialize("NNEhOL0iOcflg8f1U3HUqiCq-gzGzoHsz", "7kSmkbbb3DktmHALlShDsBUF");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>



  

  

  
  

  

  

  

</body>
</html>
